package com.web.crawler;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.web.crawler.ElasticSearchUtil;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.Map;
import java.util.stream.Stream;

public class DocumentIndexer {
    private static final ObjectMapper mapper = new ObjectMapper();

    public static void indexDocuments(String language) throws IOException {
        String folderPath = "C:\\Users\\jf\\IdeaProjects\\web-crawler\\documents\\" + language; // en 或 zh
        String indexName = "news_" + language;

        try (Stream<Path> paths = Files.walk(Paths.get(folderPath))) {
            paths.filter(Files::isRegularFile)
                    .forEach(path -> {
                        try {
                            String fileName = path.getFileName().toString();
                            String content = new String(Files.readAllBytes(path));

                            Map<String, String> document = new HashMap<>();
                            document.put("title", fileName);
                            document.put("content", content);
                            document.put("language", language);

                            String json = mapper.writeValueAsString(document);
                            String id = fileName.replace(".txt", "");

                            ElasticSearchUtil.createIndex(indexName, id, json);
                            System.out.println("Indexed: " + fileName);
                        } catch (IOException e) {
                            e.printStackTrace();
                        }
                    });
        }
    }
}